# -*- coding: utf-8 -*-
import scrapy
import datetime
from scrapy.http import Request
from urllib import parse
from ArticleSpider.items import JobboleArticleItem
from  ArticleSpider.untils.commom import to_md5

class JobboleSpider(scrapy.Spider):
    name = "jobbole"
    allowed_domains = ["blog.jobbole.com"]
    start_urls = ["http://blog.jobbole.com/all-posts/"]
    def parse(self, response):
        #response.url能获取当前的url
        post_node=response.css("#archive .post-thumb a")
        for node in post_node:
            post_url=node.css("::attr(href)").extract_first("")
            image_url=node.css("img::attr(src)").extract_first("")
            yield Request(url=parse.urljoin(response.url,post_url), meta={"image_url":image_url}, callback=self.article_detail)
            #yield Request(url=post_url, meta={"front_image_url":image_url}, callback=self.article_detail)

        next_url=response.css(".next.page-numbers::attr(href)").extract()[0]
        if next_url:
            yield Request(url=next_url,callback=self.parse)

    def article_detail(self,response):
        article_item=JobboleArticleItem()
        #下面的这个get方法不用也可以，meta是一个字典，直接用字典就可以，但是get方法，后面加一个空，可以避免异常抛出
        image_url=response.meta.get("image_url","")
        datatime=response.css("div.entry-meta p::text").extract()[0].strip().replace("·","")
        title=response.css(".entry-header h1::text").extract()[0]
        content=response.css(".entry").extract()
        zan_num=response.css(".post-adds h10::text").extract()[0]
        tag=response.css(".entry-meta .entry-meta-hide-on-mobile a::text").extract()[0]

        article_item['title']=title
        article_item['url']=response.url
        article_item['url_id']=to_md5(response.url)
        # try:
        #     datatime=datetime.datetime.strptime(datatime,"%Y/%m/&d").date()
        article_item['datatime']=datatime
        article_item['tag']=tag
        article_item['content']=content
        article_item['zan_num']=zan_num
        article_item['image_url']=[image_url]

        yield  article_item
